from __future__ import print_function
import sys
from operator import add
from pyspark.sql import SparkSession
from pyspark import SparkContext
from pyspark.sql import SQLContext
from pyspark.conf import SparkConf
from pyspark.sql import Row

print("-------------------------11111111111----------------------------------")
spark=SparkSession.builder.appName("lz").getOrCreate()
sc = SparkContext.getOrCreate()
df = spark.read.options(header='True', inferSchema='True', delimiter=',') \
  .csv("file:///workspace/bdkit-demo/spark-python-demo3/src/task3.csv")
print("1111111111111111111111111111111111")
print(type(df))
df.printSchema()
print("2222222222222222222222222222222222")
df.registerTempTable('user_table')
# 写SQL语句，用show()方法显示
sqlContext = SparkSession.builder.getOrCreate()
#sqlContext.sql("select * from user_table").show()
#sqlContext.sql("select employer_type from user_table").show()
#sqlContext.sql('SELECT distinct employer_type FROM user_table').show()
#where work_year in ('5 years','6 years','7 years','8 years','9 years','10+ years')
sqlContext.sql("SELECT user_id,censor_status,work_year FROM user_table where work_year in ('5 years','6 years','7 years','8 years','9 years','10+ years')").show()
censor_status_ratio=sqlContext.sql("SELECT user_id,censor_status,work_year FROM user_table where work_year in ('5 years','6 years','7 years','8 years','9 years','10+ years')")
print("-------------------------------------------------")
print(type(censor_status_ratio))
censor_status_ratio.repartition(1).write.csv("file:///workspace/bdkit-demo/spark-python-demo3/src/censor_status_ratio.csv",encoding="utf-8",sep=',')
